﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Net;
using System.IO;
using System.Threading;
using System.Web;
using HtmlAgilityPack;
using System.Text.RegularExpressions;

namespace AutoAdsParser
{
    class UrlDownloader
    {
        private List<AutoAdItem> _links = new List<AutoAdItem>();
        private List<AutoAdItem> _recentLinks = new List<AutoAdItem>();
        private int _maxPoolCount = 10;
        private Semaphore _pool;
        private int _maxDays = 3;
        private Random _random = new Random();
        private int _requestTimeout = 15000;
        private bool _allowNQuestions=true;

        private enum ContentTypes
        {
            ListOfAds,
            AutoAd,
            Contacts
        }

        //List of proxy http://www.hidemyass.com/proxy-list/

        public UrlDownloader()
        {
            Console.WriteLine("Prepearing proxies");
            var model = new ProxiesEntities();
            var anonymProxies = new WebClient().DownloadString("http://www.proxynova.com/proxy_list.txt?country=ru").Split(new[] { "\n" }, StringSplitOptions.RemoveEmptyEntries).Where(p => Regex.IsMatch(p, @"\d{1,3}\.")).ToArray();
            var notExistingProxies = anonymProxies.Where(p => !model.ProxyServer.Any(ps => ps.Address == p));
            foreach (var notExistingProxy in notExistingProxies)
            {
                var newProxy = new ProxyServer()
                    {
                        Id = model.ProxyServer.Any() ? model.ProxyServer.Max(i => i.Id) + 1 : 1,
                        Address = notExistingProxy,
                        CreateDate = DateTime.Now,
                        DoNotUse = false
                    };
                model.AddToProxyServer(newProxy);
                model.SaveChanges();
            }
            Console.WriteLine("Completed");
        }

        public string GetProxyAddress()
        {
            var model = new ProxiesEntities();
            var countAvailableProxy = model.ProxyServer.Count(p => !p.DoNotUse.Value);
            countAvailableProxy = model.ProxyServer.Count(p => !p.DoNotUse.Value);
            var proxyRandomNum = _random.Next(1, countAvailableProxy);
            var proxy =
                model.ProxyServer.Where(p => !p.DoNotUse.Value)
                     .OrderBy(i => i.Id)
                     .Skip(proxyRandomNum - 1)
                     .Take(1)
                     .FirstOrDefault();
            return proxy.Address;
        }

        public void DownloadUrl(string url) 
        {
            _pool = new Semaphore(_maxPoolCount, _maxPoolCount);

            var startMsg = "Starting indexing at: " + DateTime.Now;
            Program.AllAdsLogger.Info(startMsg);
            Program.RecentAdsLogger.Info(startMsg);
            
            url = url.Replace("page1", "page{0}");
            var counter=0;
            while(true){         
                counter++;

                Console.WriteLine("Get: " + string.Format(url, counter));
                var html = GetContentByUrl(string.Format(url, counter),ContentTypes.ListOfAds);
                //var html = GetContentByUrl("http://novosibirsk.drom.ru/toyota/echo/9561341.html",ContentTypes.AutoAd);
             
                HtmlDocument doc = new HtmlDocument();
                doc.LoadHtml(html);
                var linksNodes = doc.DocumentNode.SelectNodes("//nobr/a");
                if(linksNodes!=null)
                {
                    for (int i = 0; i < linksNodes.Count; i++)
                    {
                        var link = linksNodes[i];
                        var adUrl = link.GetAttributeValue("href", "");

                        _pool.WaitOne();
                        new Action<string>(ProcessAd).BeginInvoke(adUrl, null, null);
                    }
                }
                else{
                    Program.AllAdsLogger.Info(string.Join("\r\n", _links.Select(i => i.Url).ToArray()));
                    Program.AllAdsLogger.Info("Total ads count: " + _links.Count + "\r\n");
                    Program.AllAdsLogger.Info(html);

                    Program.RecentAdsLogger.Info("Total ads count: " + _recentLinks.Count + "\r\n");
                    Program.RecentAdsLogger.Info("End indexing at: " + DateTime.Now);
                    break;
                }
               
            }
            
        }

        private void ProcessAd(object adUrl)
        {
            Console.WriteLine(Thread.CurrentThread.ManagedThreadId + ", start processing: " + adUrl);
            var autoAd = new AutoAdItem() { Url = adUrl.ToString() };
            _links.Add(autoAd);

            var html = GetContentByUrl(adUrl.ToString(),ContentTypes.AutoAd);

            HtmlDocument adDoc = new HtmlDocument();
            adDoc.LoadHtml(html);

            //get ad id
            autoAd.Id=int.Parse(Regex.Match(adUrl.ToString(), @"\d{6,10}").Value);

            //get ad date
            var dateArea = adDoc.DocumentNode.SelectNodes("//p[@class='autoNum']");
            if (dateArea != null)
            {
                var dateAreaText = dateArea[0].InnerText;
                var createdDate = DateTime.Parse(Regex.Match(dateAreaText, @"\d{1,2}-\d{1,2}-\d{4}").ToString());
                autoAd.CreatedDate = createdDate;
            }

            //get price
            var priceArea = adDoc.DocumentNode.SelectNodes("//div[@class='price']");
            if (priceArea != null)
            {
                var priceText = priceArea[0].InnerText.Replace("&nbsp;","").Replace("руб.","");
                autoAd.Price = int.Parse(priceText);
            }

            //get count of views
            var countOfViewsArea = adDoc.DocumentNode.SelectNodes("//p/text()[contains(.,'Просмотров')]");
            if (countOfViewsArea != null)
            {
                var countOfViews = Regex.Match(countOfViewsArea[0].InnerText, @"\d{1,5}");
                autoAd.CountOfViews = int.Parse(countOfViews.ToString());
            }

            //car name and year
            var nameArea = adDoc.DocumentNode.SelectNodes("//h3/text()[contains(.,'год')]");
            if (nameArea != null)
            {
                var nameParts=nameArea[0].InnerText.Split(',');
                autoAd.Name =nameParts[0];
                autoAd.Year = int.Parse(nameParts[1].Replace("год","").Trim());
            }

            //questions
            var questionArea = adDoc.DocumentNode.SelectNodes("//div[@class='question']/div[@class='n']");
            if (questionArea != null)
            {
                autoAd.QuestionsCount = questionArea.Count-1;
            }

            //car sold
            var soldArea = adDoc.DocumentNode.SelectNodes("//h3/text()[contains(.,'продана')]");
            if (soldArea != null)
            {
                autoAd.Sold = true;
            }
            else {
                //get contacts
                var contactsData = GetContentByUrl(string.Format("http://novosibirsk.drom.ru/auto/?bull_id={0}&obj=show_contacts&crossdomain_ajax_request=2&request=ajax_show_contacts",autoAd.Id),ContentTypes.Contacts);
                autoAd.Contacts = Regex.Match(contactsData, @"\+\d\s\(\d{3}\)\s\d{3}\-\d{4}").Value;
            }

            if (autoAd.GetDiffDays()>=0 && autoAd.GetDiffDays()<=_maxDays && !autoAd.Sold)
            {
                Program.RecentAdsLogger.Info("views: " + autoAd.CountOfViews + ", age: " + autoAd.GetDiffDays() + " day, q: " + autoAd.QuestionsCount + ", url: " + autoAd.Url);
            }
            autoAd.Save();
            Program.AllAdsLogger.Info("Processed url:" + adUrl);
            _pool.Release();
        }

        string GetContentByUrl(string url,ContentTypes contentType) {
            var html = "";

            while (html == "" || html.IndexOf("Блокировка вашего IP") > -1)
            {
                var request = (HttpWebRequest)HttpWebRequest.Create(url);
                request.UserAgent= "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.97 Safari/537.11";
                request.ContentType = "text/plain; charset=utf-8 ";
                request.CookieContainer=new CookieContainer();
                request.Accept = "*/*";
                request.Headers.Add("Accept-Encoding", "gzip,deflate,sdch");
                request.Headers.Add("Accept-Language", "ru-RU,ru;q=0.8,en-US;q=0.6,en;q=0.4");
                request.Headers.Add("Accept-Charset", "windows-1251,utf-8;q=0.7,*;q=0.3");
                request.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;

                request.Timeout = _requestTimeout;
                var proxyAddr = GetProxyAddress();;
                Program.AllAdsLogger.Info("Send request via proxy:" + proxyAddr + ",url:" + url);
                request.Proxy = new WebProxy(proxyAddr);
                try
                {
                    using (var response = request.GetResponse())
                    using (var stream = response.GetResponseStream())
                    using (var reader = new StreamReader(stream,Encoding.GetEncoding(1251)))
                    {
                        var readFuncDelegate =new Func<string>(reader.ReadToEnd);
                        var readFuncRes= readFuncDelegate.BeginInvoke(null, null);
                        if(readFuncRes.AsyncWaitHandle.WaitOne(_requestTimeout))
                        {
                            html = readFuncDelegate.EndInvoke(readFuncRes);
                        }
                        else
                        {
                            var msg = "Read timeout: " + proxyAddr + "";
                            Console.WriteLine(msg);
                            Program.AllAdsLogger.Error(msg);
                            throw new Exception(msg);
                        }

                        if (html.IndexOf("Блокировка вашего IP") > -1)
                        {
                            var msg = "Blocked proxy: " + proxyAddr;
                            Console.WriteLine(msg);
                            Program.AllAdsLogger.Error(msg);
                            html = "";
                            throw new Exception(msg);
                        }
                        else if (!_allowNQuestions && contentType == ContentTypes.AutoAd && html.IndexOf("ask_question_wrapper") == -1)
                        {
                            var msg = "Blocked proxy: " + proxyAddr + ", no questions returned";
                            Console.WriteLine(msg);
                            Program.AllAdsLogger.Error(msg);
                            html = "";
                            throw new Exception(msg);
                        }
                    }
                }
                catch (Exception ex)
                {
                    var model = new ProxiesEntities();
                    var proxy = model.ProxyServer.FirstOrDefault(i => i.Address == proxyAddr);
                    if (proxy!=null)
                    {
                        proxy.CountFails = proxy.CountFails.HasValue ? proxy.CountFails.Value + 1 : 1;
                        if (proxy.CountFails >= 1)
                            proxy.DoNotUse = true;

                        model.SaveChanges();
                    }
                    Console.WriteLine("Request error, proxy: " + proxyAddr + ", " + ex.Message);
                    Program.AllAdsLogger.Error("Request error, proxy: " + proxyAddr, ex);
                }
            }
            return html;
        }
    }
}
